#include"../include/DictProducer.hpp"
#include"../include/cppjieba/Jieba.hpp"
#include"../include/SplitTool.hpp"
#include<dirent.h>//DIR和opendir的头文件
#include<sys/types.h>//DIR和opendir的头文件
#include<unistd.h>//chdir,getcwd
#include<sys/stat.h>//stat
#include<string.h>

#include<string>
#include<fstream>
#include<sstream>                                                        

using std::cout;
using std::endl;
using std::ifstream;
using std::istringstream;
using std::ofstream;
using std::cerr;

using namespace wd;

DictProducer::DictProducer(const string& dir)
:_dir(dir)
,_LineNum(getDictline())
{//根据传进来的目录，读目录里的英文单词
    getStopWordList();
    getFiles();
    buildEnDict();
    storeDict("../data/dict.dat");
}

DictProducer::DictProducer(const string& dir, SplitTool* splitTool)
:_dir(dir)
,_splitTool(splitTool)
,_LineNum(getDictline())
{  
    getStopWordList();
    getFiles();
    buildCnDict();
    storeDict("../data/dict.dat");
}

size_t DictProducer::getDictline(){
    ifstream ifs("../data/dict.dat");
    if(!ifs.good()){
        ifs.close();
        return 0;
    }
    string line;
    size_t fileLine = 0;
    while(!ifs.eof()){
        getline(ifs,line);
        ++fileLine;
    }
    ifs.close();
    return fileLine;
}

void DictProducer::getFiles(){
    DIR* pdir;
    pdir = opendir(_dir.c_str());
    if(pdir == NULL){
        cout << "opendir error" <<endl;
        return;
    }
    struct dirent* pdirent;
    while((pdirent = readdir(pdir)) != NULL){
        if(strcmp(pdirent->d_name,".") == 0 || 
           strcmp(pdirent->d_name,"..") == 0){
            continue;
        }
        else if(pdirent->d_type == DT_DIR){
            string file;
            file = _dir;
            file += '/';
            file += pdirent->d_name;
            _files.push_back(file);
        }
        else{
            string file;
            file = _dir;
            file += pdirent->d_name;
            _files.push_back(file);
        }
    }
    closedir(pdir);
    return;
}

void DictProducer::storeDict(const char* filepath){//把_dict存到文件路径下去
    ofstream ofs(filepath,std::ios::app);
    if(!ofs)
    {
        cerr << "ofstream error!" << endl;
        return;
    }
    for(auto & elem: _dict){
        ofs << elem.first<<" "<<elem.second << std::endl;
    }
    std::cout << "end storeDict " << std::endl;
}


void DictProducer::buildEnDict(){
    for(size_t i = 0;i < _files.size();++i){
        string filepath = _files[i];
        pushEnglishDict(filepath);
    }
    for(auto& elem : _wordFreqMap){
        _dict.push_back({elem.first,elem.second}); 
    }
}


void DictProducer::buildCnDict(){
    for(size_t i = 0;i < _files.size();++i){
        string filepath = _files[i];
        pushChineseDict(filepath);
    }
    for(auto& elem : _wordFreqMap){
        _dict.push_back({elem.first,elem.second}); 
    }
}

void DictProducer::pushChineseDict(const string& filepath){
    ifstream ifs(filepath);
    string line;
    while(getline(ifs,line)){
        vector<string> words;
        words = _splitTool->cut(line);
        
        for(auto& word : words){
            string tagWord = _splitTool->tag(word);
            if(tagWord != "x" && tagWord != "eng" && tagWord != "m"){//如果不是停用词
                if(_wordFreqMap[word]){
                    ++_wordFreqMap[word];
                }
                else{
                    _wordFreqMap[word] = 1;
                }
            }
        }//endof while(iss >> word)
    }
    ifs.close();
}

void DictProducer::pushEnglishDict(const string& filepath){
    ifstream ifs(filepath);
    string word;
    string line;
    while(getline(ifs,line)){
        string newline = dealLine(line);////"ASddsa;faae"-->"asddsa faae"
        std::istringstream iss(newline);//没有标点符号和大写字母的一行
        while(iss >> word){
            if(_stopWordList.find(word) == _stopWordList.end()){//如果不是停用词
                if(_wordFreqMap[word]){
                    ++_wordFreqMap[word];
                }
                else{
                    _wordFreqMap[word] = 1;
                }
            }
        }//endof while(iss>>word)
    }//endof while(getline)
    std::cout << "pushEnglishDict" << std::endl;

    ifs.close();
}

string DictProducer::dealLine(string& line){
    string newstr;
    for(size_t idx = 0;idx < line.size();++idx){
        if(isalpha(line[idx])){//如果是字母
            if(isupper(line[idx])){//如果是大写字母，转小写
                newstr += line[idx] + 32;
            }
            else{
                newstr += line[idx];
            }
        }
        else{//不是字母，则设为空格
            newstr += " ";
        }
    }//endof for
    //   std::cout << "dealLine " << std::endl;
    return newstr;
}

void DictProducer::getStopWordList(){
    ifstream ifs("../conf/yuliao/stop_words_eng.txt");//英文停用词
    string word;
    while(getline(ifs,word)){
        _stopWordList.insert(word);
    }   
    ifs.close();

    ifstream ifsc("../conf/yuliao/stop_words_zh.txt",std::ios::app);//中文停用词
    string wordc;
    while(getline(ifsc,wordc)){
        _stopWordList.insert(wordc);
    }   
    ifsc.close();
    std::cout << "getStopWordList" << std::endl;

}

map<string,set<int>> DictProducer::getIndex(){
    return _index;
}

void DictProducer::doIndexes(const string& indFilePath){
    /* cout << "文件已有" << fileLine << "行" << endl; */
    for(size_t idx=0;idx < _dict.size();++idx){//取完_dict中的单词
        if((_dict[idx].first[0]) >= 'a' && _dict[idx].first[0] <= 'z'){//为英文
            string str = "a";
            char ch = str[0];
            while(islower(ch)){
                string str = _dict[idx].first;
                for(size_t i = 0;i < str.size();++i){//取一个词的所有字母
                    if(str[i] == ch){//字母与26字母中的一个相同
                        string s;
                        s = ch;
                        _index[s].insert(idx+_LineNum);
                    }
                }//endof 取字母
                ++ch;
            }
        }//endof en
        else{//如果是中文
            string str = _dict[idx].first;
            char ch = str[0];
            int nBytes = 1;
            if(ch & (1 << 7)){//最高位为1
                for(int idx2 = 0;idx2 != 6;++idx2){
                    if(ch & (1 << (6 - idx2))){
                        ++nBytes;
                    }
                    else{
                        break;
                    }
                }//endof for
            }//已得出中文一个字符占多少字节
            for(size_t i = 0;i < str.size();i += nBytes){
                string ins = str.substr(i,nBytes);
                _index[ins].insert(idx+_LineNum);
            }
        }//endof chinese
    }//endof all for
    ofstream ofs(indFilePath,std::ios::app);
    if(!ofs){
        cerr << "ofstream error" << endl;
        return;
    }
    for(auto & elem : _index){
        ofs << elem.first << " ";
        for(auto & setInt : elem.second){
            ofs << setInt << " ";
        }
        ofs << endl;
    }

    std::cout << "doIndexes " << std::endl;
}

